{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/xlnet/model_utils.py:295: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xlnet\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "import model_utils\n",
    "import pickle\n",
    "import json\n",
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "from prepro_utils import preprocess_text, encode_ids\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('xlnet-base-29-03-2020/sp10m.cased.v9.model')\n",
    "\n",
    "def tokenize_fn(text):\n",
    "    text = preprocess_text(text, lower= False)\n",
    "    return encode_ids(sp_model, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEG_ID_A   = 0\n",
    "SEG_ID_B   = 1\n",
    "SEG_ID_CLS = 2\n",
    "SEG_ID_SEP = 3\n",
    "SEG_ID_PAD = 4\n",
    "\n",
    "special_symbols = {\n",
    "    \"<unk>\"  : 0,\n",
    "    \"<s>\"    : 1,\n",
    "    \"</s>\"   : 2,\n",
    "    \"<cls>\"  : 3,\n",
    "    \"<sep>\"  : 4,\n",
    "    \"<pad>\"  : 5,\n",
    "    \"<mask>\" : 6,\n",
    "    \"<eod>\"  : 7,\n",
    "    \"<eop>\"  : 8,\n",
    "}\n",
    "\n",
    "VOCAB_SIZE = 32000\n",
    "UNK_ID = special_symbols[\"<unk>\"]\n",
    "CLS_ID = special_symbols[\"<cls>\"]\n",
    "SEP_ID = special_symbols[\"<sep>\"]\n",
    "MASK_ID = special_symbols[\"<mask>\"]\n",
    "EOD_ID = special_symbols[\"<eod>\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('train_X.pkl', 'rb') as fopen:\n",
    "    train_X, train_Y, train_depends, train_segments, train_masks = pickle.load(fopen)\n",
    "    \n",
    "with open('test_X.pkl', 'rb') as fopen:\n",
    "    test_X, test_Y, test_depends, test_segments, test_masks = pickle.load(fopen)\n",
    "    \n",
    "with open('tags.pkl', 'rb') as fopen:\n",
    "    idx2tag, tag2idx = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xlnet\n",
    "import model_utils\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "kwargs = dict(\n",
    "      is_training=True,\n",
    "      use_tpu=False,\n",
    "      use_bfloat16=False,\n",
    "      dropout=0.1,\n",
    "      dropatt=0.1,\n",
    "      init='normal',\n",
    "      init_range=0.1,\n",
    "      init_std=0.05,\n",
    "      clamp_len=-1)\n",
    "\n",
    "xlnet_parameters = xlnet.RunConfig(**kwargs)\n",
    "xlnet_config = xlnet.XLNetConfig(json_path='xlnet-base-29-03-2020/config.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1173456 117345\n"
     ]
    }
   ],
   "source": [
    "epoch = 3\n",
    "batch_size = 8\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(train_X) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)\n",
    "print(num_train_steps, num_warmup_steps)\n",
    "\n",
    "training_parameters = dict(\n",
    "      decay_method = 'poly',\n",
    "      train_steps = num_train_steps,\n",
    "      learning_rate = 2e-5,\n",
    "      warmup_steps = num_warmup_steps,\n",
    "      min_lr_ratio = 0.0,\n",
    "      weight_decay = 0.00,\n",
    "      adam_epsilon = 1e-8,\n",
    "      num_core_per_host = 1,\n",
    "      lr_layer_decay_rate = 1,\n",
    "      use_tpu=False,\n",
    "      use_bfloat16=False,\n",
    "      dropout=0.0,\n",
    "      dropatt=0.0,\n",
    "      init='normal',\n",
    "      init_range=0.1,\n",
    "      init_std=0.02,\n",
    "      clip = 1.0,\n",
    "      clamp_len=-1,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Parameter:\n",
    "    def __init__(self, decay_method, warmup_steps, weight_decay, adam_epsilon, \n",
    "                num_core_per_host, lr_layer_decay_rate, use_tpu, learning_rate, train_steps,\n",
    "                min_lr_ratio, clip, **kwargs):\n",
    "        self.decay_method = decay_method\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.weight_decay = weight_decay\n",
    "        self.adam_epsilon = adam_epsilon\n",
    "        self.num_core_per_host = num_core_per_host\n",
    "        self.lr_layer_decay_rate = lr_layer_decay_rate\n",
    "        self.use_tpu = use_tpu\n",
    "        self.learning_rate = learning_rate\n",
    "        self.train_steps = train_steps\n",
    "        self.min_lr_ratio = min_lr_ratio\n",
    "        self.clip = clip\n",
    "        \n",
    "training_parameters = Parameter(**training_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiAAttention:\n",
    "    def __init__(self, input_size_encoder, input_size_decoder, num_labels):\n",
    "        self.input_size_encoder = input_size_encoder\n",
    "        self.input_size_decoder = input_size_decoder\n",
    "        self.num_labels = num_labels\n",
    "        \n",
    "        self.W_d = tf.get_variable(\"W_d\", shape=[self.num_labels, self.input_size_decoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_e = tf.get_variable(\"W_e\", shape=[self.num_labels, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.U = tf.get_variable(\"U\", shape=[self.num_labels, self.input_size_decoder, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        \n",
    "    def forward(self, input_d, input_e, mask_d=None, mask_e=None):\n",
    "        batch = tf.shape(input_d)[0]\n",
    "        length_decoder = tf.shape(input_d)[1]\n",
    "        length_encoder = tf.shape(input_e)[1]\n",
    "        out_d = tf.expand_dims(tf.matmul(self.W_d, tf.transpose(input_d, [0, 2, 1])), 3)\n",
    "        out_e = tf.expand_dims(tf.matmul(self.W_e, tf.transpose(input_e, [0, 2, 1])), 2)\n",
    "        output = tf.matmul(tf.expand_dims(input_d, 1), self.U)\n",
    "        output = tf.matmul(output, tf.transpose(tf.expand_dims(input_e, 1), [0, 1, 3, 2]))\n",
    "        \n",
    "        output = output + out_d + out_e\n",
    "        \n",
    "        if mask_d is not None:\n",
    "            d = tf.expand_dims(tf.expand_dims(mask_d, 1), 3)\n",
    "            e = tf.expand_dims(tf.expand_dims(mask_e, 1), 2)\n",
    "            output = output * d * e\n",
    "            \n",
    "        return output\n",
    "    \n",
    "class BiLinear:\n",
    "    def __init__(self, left_features, right_features, out_features):\n",
    "        self.left_features = left_features\n",
    "        self.right_features = right_features\n",
    "        self.out_features = out_features\n",
    "        \n",
    "        self.U = tf.get_variable(\"U-bi\", shape=[out_features, left_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_l = tf.get_variable(\"Wl\", shape=[out_features, left_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_r = tf.get_variable(\"Wr\", shape=[out_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    def forward(self, input_left, input_right):\n",
    "        left_size = tf.shape(input_left)\n",
    "        output_shape = tf.concat([left_size[:-1], [self.out_features]], axis = 0)\n",
    "        batch = tf.cast(tf.reduce_prod(left_size[:-1]), tf.int32)\n",
    "        input_left = tf.reshape(input_left, (batch, self.left_features))\n",
    "        input_right = tf.reshape(input_right, (batch, self.right_features))\n",
    "        tiled = tf.tile(tf.expand_dims(input_left, axis = 0), (self.out_features,1,1))\n",
    "        output = tf.transpose(tf.reduce_sum(tf.matmul(tiled, self.U), axis = 2))\n",
    "        output = output + tf.matmul(input_left, tf.transpose(self.W_l))\\\n",
    "        + tf.matmul(input_right, tf.transpose(self.W_r))\n",
    "        \n",
    "        return tf.reshape(output, output_shape)\n",
    "\n",
    "\n",
    "    \n",
    "    def decode(self, input_word, input_char, mask, leading_symbolic=0):\n",
    "        out_arc, out_type, _ = self.forward(input_word, input_char, mask)\n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        out_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        out_arc = tf.where(minus_mask, tf.fill(tf.shape(out_arc), -np.inf), out_arc)\n",
    "        heads = tf.argmax(out_arc, axis = 1)\n",
    "        type_h, type_c = out_type\n",
    "        batch = tf.shape(type_h)[0]\n",
    "        max_len = tf.shape(type_h)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.cast(tf.transpose(heads), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        out_type = out_type[:, :, leading_symbolic:]\n",
    "        types = tf.argmax(out_type, axis = 2)\n",
    "        return heads, types\n",
    "    \n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        learning_rate,\n",
    "        hidden_size_word,\n",
    "        cov = 0.0):\n",
    "        \n",
    "        self.words = tf.placeholder(tf.int32, (None, None))\n",
    "        self.segment_ids = tf.placeholder(tf.int32, [None, None])\n",
    "        self.input_masks = tf.placeholder(tf.float32, [None, None])\n",
    "        self.heads = tf.placeholder(tf.int32, (None, None))\n",
    "        self.types = tf.placeholder(tf.int32, (None, None))\n",
    "        self.switch = tf.placeholder(tf.bool, None)\n",
    "        self.mask = tf.cast(tf.math.not_equal(self.words, 0), tf.float32)\n",
    "        self.maxlen = tf.shape(self.words)[1]\n",
    "        self.lengths = tf.count_nonzero(self.words, 1)\n",
    "        mask = self.mask\n",
    "        heads = self.heads\n",
    "        types = self.types\n",
    "        \n",
    "        self.arc_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.arc_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.attention = BiAAttention(hidden_size_word, hidden_size_word, 1)\n",
    "\n",
    "        self.type_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.type_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.bilinear = BiLinear(hidden_size_word, hidden_size_word, len(tag2idx))\n",
    "        \n",
    "        xlnet_model = xlnet.XLNetModel(\n",
    "            xlnet_config=xlnet_config,\n",
    "            run_config=xlnet_parameters,\n",
    "            input_ids=tf.transpose(self.words, [1, 0]),\n",
    "            seg_ids=tf.transpose(self.segment_ids, [1, 0]),\n",
    "            input_mask=tf.transpose(self.input_masks, [1, 0]))\n",
    "        output_layer = xlnet_model.get_sequence_output()\n",
    "        output_layer = tf.transpose(output_layer, [1, 0, 2])\n",
    "        \n",
    "        arc_h = tf.nn.elu(self.arc_h(output_layer))\n",
    "        arc_c = tf.nn.elu(self.arc_c(output_layer))\n",
    "        \n",
    "        type_h = tf.nn.elu(self.type_h(output_layer))\n",
    "        type_c = tf.nn.elu(self.type_c(output_layer))\n",
    "        \n",
    "        out_arc = tf.squeeze(self.attention.forward(arc_h, arc_c, mask_d=self.mask, \n",
    "                                                    mask_e=self.mask), axis = 1)\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        \n",
    "        decode_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        decode_arc = tf.where(minus_mask, tf.fill(tf.shape(decode_arc), -np.inf), decode_arc)\n",
    "        self.heads_seq = tf.argmax(decode_arc, axis = 1)\n",
    "        self.heads_seq = tf.identity(self.heads_seq, name = 'heads_seq')\n",
    "        \n",
    "        t = tf.cast(tf.transpose(self.heads_seq), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        self.tags_seq = tf.argmax(out_type, axis = 2)\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'tags_seq')\n",
    "        \n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            out_type, self.types, self.lengths\n",
    "        )\n",
    "        crf_loss = tf.reduce_mean(-log_likelihood)\n",
    "        self.logits, _ = tf.contrib.crf.crf_decode(\n",
    "            out_type, transition_params, self.lengths\n",
    "        )\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        minus_inf = -1e8\n",
    "        minus_mask = (1 - mask) * minus_inf\n",
    "        out_arc = out_arc + tf.expand_dims(minus_mask, axis = 2) + tf.expand_dims(minus_mask, axis = 1)\n",
    "        loss_arc = tf.nn.log_softmax(out_arc, dim=1)\n",
    "        loss_type = tf.nn.log_softmax(out_type, dim=2)\n",
    "        loss_arc = loss_arc * tf.expand_dims(mask, axis = 2) * tf.expand_dims(mask, axis = 1)\n",
    "        loss_type = loss_type * tf.expand_dims(mask, axis = 2)\n",
    "        num = tf.reduce_sum(mask) - tf.cast(batch, tf.float32)\n",
    "        child_index = tf.tile(tf.expand_dims(tf.range(0, max_len), 1), [1, batch])\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0)], axis = 0))\n",
    "        loss_arc = tf.gather_nd(loss_arc, concatenated)\n",
    "        loss_arc = tf.transpose(loss_arc, [1, 0])\n",
    "        \n",
    "        t = tf.transpose(types)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        loss_type = tf.gather_nd(loss_type, concatenated)\n",
    "        loss_type = tf.transpose(loss_type, [1, 0])\n",
    "        cost = (tf.reduce_sum(-loss_arc) / num) + (tf.reduce_sum(-loss_type) / num)\n",
    "        \n",
    "        self.cost = tf.cond(self.switch, lambda: cost + crf_loss, lambda: cost)\n",
    "        self.optimizer = tf.train.AdamOptimizer(\n",
    "            learning_rate = learning_rate\n",
    "        ).minimize(self.cost)\n",
    "        \n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        \n",
    "        self.prediction = tf.boolean_mask(self.logits, mask)\n",
    "        mask_label = tf.boolean_mask(self.types, mask)\n",
    "        correct_pred = tf.equal(tf.cast(self.prediction, tf.int32), mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        self.prediction = tf.cast(tf.boolean_mask(self.heads_seq, mask), tf.int32)\n",
    "        mask_label = tf.boolean_mask(self.heads, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy_depends = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:memory input None\n",
      "INFO:tensorflow:Use float type <dtype: 'float32'>\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import re\n",
    "\n",
    "def get_assignment_map_from_checkpoint(tvars, init_checkpoint):\n",
    "    \"\"\"Compute the union of the current variables and checkpoint variables.\"\"\"\n",
    "    assignment_map = {}\n",
    "    initialized_variable_names = {}\n",
    "\n",
    "    name_to_variable = collections.OrderedDict()\n",
    "    for var in tvars:\n",
    "        name = var.name\n",
    "        m = re.match('^(.*):\\\\d+$', name)\n",
    "        if m is not None:\n",
    "            name = m.group(1)\n",
    "        name_to_variable[name] = var\n",
    "\n",
    "    init_vars = tf.train.list_variables(init_checkpoint)\n",
    "\n",
    "    assignment_map = collections.OrderedDict()\n",
    "    for x in init_vars:\n",
    "        (name, var) = (x[0], x[1])\n",
    "        if name not in name_to_variable:\n",
    "            continue\n",
    "        assignment_map[name] = name_to_variable[name]\n",
    "        initialized_variable_names[name] = 1\n",
    "        initialized_variable_names[name + ':0'] = 1\n",
    "\n",
    "    return (assignment_map, initialized_variable_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tvars = tf.trainable_variables()\n",
    "checkpoint = 'xlnet-base-29-03-2020/model.ckpt-300000'\n",
    "assignment_map, initialized_variable_names = get_assignment_map_from_checkpoint(tvars, \n",
    "                                                                                checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from xlnet-base-29-03-2020/model.ckpt-300000\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver(var_list = assignment_map)\n",
    "saver.restore(sess, checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "batch_x = train_X[:5]\n",
    "batch_x = pad_sequences(batch_x,padding='post')\n",
    "batch_y = train_Y[:5]\n",
    "batch_y = pad_sequences(batch_y,padding='post')\n",
    "batch_depends = train_depends[:5]\n",
    "batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "batch_segments = train_segments[:5]\n",
    "batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "batch_masks = train_masks[:5]\n",
    "batch_masks = pad_sequences(batch_masks, padding='post', value = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.03529412, 0.07058824, 140.91621]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.segment_ids: batch_segments,\n",
    "                model.input_masks: batch_masks,\n",
    "                model.switch: False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.047058824, 0.023529412, 352.61438]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.segment_ids: batch_segments,\n",
    "                model.input_masks: batch_masks,\n",
    "                model.switch: True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 391153/391153 [20:11:00<00:00,  5.38it/s, accuracy=0.867, accuracy_depends=0.733, cost=0.328]    \n",
      "test minibatch loop: 100%|██████████| 1250/1250 [02:16<00:00,  9.19it/s, accuracy=0.945, accuracy_depends=0.853, cost=0.234] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 0.401834, training acc: 0.901875, training depends: 0.845306, valid loss: 0.386272, valid acc: 0.902461, valid depends: 0.848041\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 1\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        batch_segments = train_segments[i: index]\n",
    "        batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "        batch_masks = train_masks[i: index]\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "        \n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.segment_ids: batch_segments,\n",
    "                    model.input_masks: batch_masks,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        batch_segments = test_segments[i: index]\n",
    "        batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "        batch_masks = test_masks[i: index]\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "        \n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.segment_ids: batch_segments,\n",
    "                    model.input_masks: batch_masks,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'xlnet-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 391153/391153 [19:55:40<00:00,  5.45it/s, accuracy=1, accuracy_depends=0.733, cost=0.617]       \n",
      "test minibatch loop: 100%|██████████| 1250/1250 [02:17<00:00,  9.12it/s, accuracy=0.969, accuracy_depends=0.859, cost=1.82] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 3.694703, training acc: 0.950186, training depends: 0.846899, valid loss: 3.673369, valid acc: 0.951140, valid depends: 0.844435\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 1\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        batch_segments = train_segments[i: index]\n",
    "        batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "        batch_masks = train_masks[i: index]\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "        \n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.segment_ids: batch_segments,\n",
    "                    model.input_masks: batch_masks,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        batch_segments = test_segments[i: index]\n",
    "        batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "        batch_masks = test_masks[i: index]\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "        \n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.segment_ids: batch_segments,\n",
    "                    model.input_masks: batch_masks,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'xlnet-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8, 52), (8, 52))"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_x.shape, batch_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_sentencepiece_tokens_tagging(x, y):\n",
    "    new_paired_tokens = []\n",
    "    n_tokens = len(x)\n",
    "    rejected = ['<cls>', '<sep>']\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    while i < n_tokens:\n",
    "\n",
    "        current_token, current_label = x[i], y[i]\n",
    "        if not current_token.startswith('▁') and current_token not in rejected:\n",
    "            previous_token, previous_label = new_paired_tokens.pop()\n",
    "            merged_token = previous_token\n",
    "            merged_label = [previous_label]\n",
    "            while (\n",
    "                not current_token.startswith('▁')\n",
    "                and current_token not in rejected\n",
    "            ):\n",
    "                merged_token = merged_token + current_token.replace('▁', '')\n",
    "                merged_label.append(current_label)\n",
    "                i = i + 1\n",
    "                current_token, current_label = x[i], y[i]\n",
    "            merged_label = merged_label[0]\n",
    "            new_paired_tokens.append((merged_token, merged_label))\n",
    "\n",
    "        else:\n",
    "            new_paired_tokens.append((current_token, current_label))\n",
    "            i = i + 1\n",
    "\n",
    "    words = [\n",
    "        i[0].replace('▁', '')\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in ['<cls>', '<sep>']\n",
    "    ]\n",
    "    labels = [i[1] for i in new_paired_tokens if i[0] not in ['<cls>', '<sep>']]\n",
    "    return words, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from malaya.function.parse_dependency import DependencyGraph\n",
    "\n",
    "PUNCTUATION = '!\"#$%&\\'()*+,./:;<=>?@[\\]^_`{|}~'\n",
    "\n",
    "def transformer_textcleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    string = re.sub('\\\\(dot\\\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    string = ' '.join(string)\n",
    "    string = re.sub(f'([{PUNCTUATION}])', r' \\1 ', string)\n",
    "    string = re.sub('\\s{2,}', ' ', string)\n",
    "    original_string = string.split()\n",
    "    string = [\n",
    "        (original_string[no], word.title() if word.isupper() else word)\n",
    "        for no, word in enumerate(string.split())\n",
    "        if len(word)\n",
    "    ]\n",
    "    return [s[0] for s in string], [s[1] for s in string]\n",
    "\n",
    "def parse_X(left):\n",
    "    left = ' '.join(left)\n",
    "    bert_tokens = tokenize_fn(left)\n",
    "    bert_tokens.extend([3, 4])\n",
    "    segment = [0] * (len(bert_tokens) - 1) + [SEG_ID_CLS]\n",
    "    input_mask = [0] * len(segment)\n",
    "    s_tokens = [sp_model.IdToPiece(i) for i in bert_tokens]\n",
    "    return bert_tokens, segment, input_mask, s_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dependency_graph(tagging, indexing):\n",
    "    \"\"\"\n",
    "    Return helper object for dependency parser results. Only accept tagging and indexing outputs from dependency models.\n",
    "    \"\"\"\n",
    "    result = []\n",
    "    for i in range(len(tagging)):\n",
    "        result.append(\n",
    "            '%d\\t%s\\t_\\t_\\t_\\t_\\t%d\\t%s\\t_\\t_'\n",
    "            % (i + 1, tagging[i][0], int(indexing[i][1]), tagging[i][1])\n",
    "        )\n",
    "    return DependencyGraph('\\n'.join(result), top_relation_label='root')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"96pt\" height=\"305pt\"\n",
       " viewBox=\"0.00 0.00 96.00 305.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 301)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-301 92,-301 92,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (makan)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M44,-260.9735C44,-249.1918 44,-233.5607 44,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"47.5001,-220.0033 44,-210.0034 40.5001,-220.0034 47.5001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"59\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (husein)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;1 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M44,-173.9735C44,-162.1918 44,-146.5607 44,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"47.5001,-133.0033 44,-123.0034 40.5001,-133.0034 47.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"64\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (ayam)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M44,-86.9735C44,-75.1918 44,-59.5607 44,-46.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"47.5001,-46.0033 44,-36.0034 40.5001,-46.0034 47.5001,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"55\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f0911678828>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'husein makan ayam'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, segment_sequence, mask_sequence, xlnet_sequence = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "            model.segment_ids: [segment_sequence],\n",
    "            model.input_masks: [mask_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(xlnet_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(xlnet_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"2221pt\" height=\"1001pt\"\n",
       " viewBox=\"0.00 0.00 2220.50 1001.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 997)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-997 2216.5,-997 2216.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"674.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"674.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (Kuala)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M674.5,-956.9735C674.5,-945.1918 674.5,-929.5607 674.5,-916.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"678.0001,-916.0033 674.5,-906.0034 671.0001,-916.0034 678.0001,-916.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"689.5\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">root</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"473.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (Lumpur)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M633.5513,-872.1224C618.1072,-866.0028 600.4272,-858.8393 584.5,-852 563.4831,-842.9751 540.4734,-832.5026 520.8804,-823.4039\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"522.1027,-820.1121 511.5602,-819.0601 519.1456,-826.4569 522.1027,-820.1121\"/>\n",
       "<text text-anchor=\"middle\" x=\"596.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"567.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (:)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M651.5589,-869.746C644.4704,-864.0791 636.6482,-857.7988 629.5,-852 618.9095,-843.4087 607.3575,-833.9346 597.1261,-825.5075\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"599.1893,-822.6724 589.2474,-819.0105 594.7358,-828.073 599.1893,-822.6724\"/>\n",
       "<text text-anchor=\"middle\" x=\"650\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 13 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>13</title>\n",
       "<text text-anchor=\"middle\" x=\"674.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">13 (membidas)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;13 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1&#45;&gt;13</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M674.5,-869.9735C674.5,-858.1918 674.5,-842.5607 674.5,-829.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"678.0001,-829.0033 674.5,-819.0034 671.0001,-829.0034 678.0001,-829.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"708\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">parataxis</text>\n",
       "</g>\n",
       "<!-- 14 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>14</title>\n",
       "<text text-anchor=\"middle\" x=\"815.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">14 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;14 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>1&#45;&gt;14</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M713.0671,-869.8567C723.7615,-864.4481 735.2461,-858.2791 745.5,-852 758.6045,-843.9754 772.4016,-834.2302 784.25,-825.417\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"786.6163,-828.0163 792.4953,-819.2023 782.403,-822.4264 786.6163,-828.0163\"/>\n",
       "<text text-anchor=\"middle\" x=\"778.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 39 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>39</title>\n",
       "<text text-anchor=\"middle\" x=\"1095.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">39 (berkata)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;39 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>1&#45;&gt;39</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M715.305,-877.9036C726.38,-875.2462 738.383,-872.4416 749.5,-870 848.5758,-848.2405 964.2603,-825.7585 1033.6046,-812.5946\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1034.3184,-816.0217 1043.492,-810.7212 1033.0152,-809.1441 1034.3184,-816.0217\"/>\n",
       "<text text-anchor=\"middle\" x=\"914\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>4</title>\n",
       "<text text-anchor=\"middle\" x=\"210.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">4 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;4 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>13&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M612.9923,-785.1082C609.7871,-784.3735 606.6086,-783.666 603.5,-783 481.7155,-756.9094 337.1463,-733.4323 261.867,-721.7669\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"262.131,-718.2663 251.7142,-720.2002 261.0634,-725.1844 262.131,-718.2663\"/>\n",
       "<text text-anchor=\"middle\" x=\"527.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 33 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>33</title>\n",
       "<text text-anchor=\"middle\" x=\"581.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">33 (melaksanakan)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;33 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>13&#45;&gt;33</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M655.2303,-782.9735C641.6385,-770.2586 623.2541,-753.0603 608.2437,-739.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"610.4388,-736.279 600.745,-732.0034 605.6567,-741.3909 610.4388,-736.279\"/>\n",
       "<text text-anchor=\"middle\" x=\"657.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">ccomp</text>\n",
       "</g>\n",
       "<!-- 37 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>37</title>\n",
       "<text text-anchor=\"middle\" x=\"702.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">37 (.)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;37 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>13&#45;&gt;37</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M680.3016,-782.9735C684.131,-771.0751 689.2239,-755.2508 693.5655,-741.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"696.9738,-742.5948 696.7058,-732.0034 690.3104,-740.4502 696.9738,-742.5948\"/>\n",
       "<text text-anchor=\"middle\" x=\"711\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 16 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>16</title>\n",
       "<text text-anchor=\"middle\" x=\"815.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16 (Seri)</text>\n",
       "</g>\n",
       "<!-- 14&#45;&gt;16 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>14&#45;&gt;16</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M815.5,-782.9735C815.5,-771.1918 815.5,-755.5607 815.5,-742.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"819.0001,-742.0033 815.5,-732.0034 812.0001,-742.0034 819.0001,-742.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"827.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 38 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>38</title>\n",
       "<text text-anchor=\"middle\" x=\"1049.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">38 (Beliau)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;38 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>39&#45;&gt;38</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1085.9688,-782.9735C1079.5543,-770.8418 1070.9818,-754.6287 1063.7599,-740.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1066.7874,-739.2077 1059.019,-732.0034 1060.5992,-742.4797 1066.7874,-739.2077\"/>\n",
       "<text text-anchor=\"middle\" x=\"1094.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 46 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>46</title>\n",
       "<text text-anchor=\"middle\" x=\"1171.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">46 (memetik)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;46 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>39&#45;&gt;46</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1111.2472,-782.9735C1122.1508,-770.4919 1136.8284,-753.6899 1148.967,-739.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1151.8299,-741.8371 1155.7729,-732.0034 1146.5581,-737.2319 1151.8299,-741.8371\"/>\n",
       "<text text-anchor=\"middle\" x=\"1152.5\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 64 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>64</title>\n",
       "<text text-anchor=\"middle\" x=\"1627.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">64 (berkata)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;64 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>39&#45;&gt;64</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1147.9662,-792.42C1246.7519,-776.2652 1459.8982,-741.4086 1565.546,-724.1316\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1566.1229,-727.5838 1575.4269,-722.5157 1564.9931,-720.6756 1566.1229,-727.5838\"/>\n",
       "<text text-anchor=\"middle\" x=\"1406\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>5</title>\n",
       "<text text-anchor=\"middle\" x=\"62.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">5 (Penerangan)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;5 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M179.8343,-695.9735C157.212,-682.6753 126.2472,-664.473 101.8077,-650.1065\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"103.521,-647.0538 93.1264,-645.0034 99.9736,-653.0884 103.521,-647.0538\"/>\n",
       "<text text-anchor=\"middle\" x=\"157.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 8 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>8</title>\n",
       "<text text-anchor=\"middle\" x=\"210.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;8 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>4&#45;&gt;8</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M210.5,-695.9735C210.5,-684.1918 210.5,-668.5607 210.5,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"214.0001,-655.0033 210.5,-645.0034 207.0001,-655.0034 214.0001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"232\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>6</title>\n",
       "<text text-anchor=\"middle\" x=\"52.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">6 (Bersatu)</text>\n",
       "</g>\n",
       "<!-- 5&#45;&gt;6 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>5&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M60.428,-608.9735C59.0738,-597.1918 57.2771,-581.5607 55.7366,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"59.1885,-567.5383 54.5694,-558.0034 52.2342,-568.3376 59.1885,-567.5383\"/>\n",
       "<text text-anchor=\"middle\" x=\"70.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 7 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>7</title>\n",
       "<text text-anchor=\"middle\" x=\"146.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">7 (,)</text>\n",
       "</g>\n",
       "<!-- 8&#45;&gt;7 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>8&#45;&gt;7</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M189.7202,-608.9921C184.0789,-603.5383 178.229,-597.3119 173.5,-591 168.0599,-583.7389 163.0898,-575.2327 158.938,-567.2711\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"162.0522,-565.673 154.461,-558.2822 155.7864,-568.7938 162.0522,-565.673\"/>\n",
       "<text text-anchor=\"middle\" x=\"194\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 9 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>9</title>\n",
       "<text text-anchor=\"middle\" x=\"227.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">9 (Wan)</text>\n",
       "</g>\n",
       "<!-- 8&#45;&gt;9 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>8&#45;&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M214.0224,-608.9735C216.3246,-597.1918 219.379,-581.5607 221.9978,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"225.4993,-568.489 223.9821,-558.0034 218.6292,-567.1465 225.4993,-568.489\"/>\n",
       "<text text-anchor=\"middle\" x=\"232.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 10 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>10</title>\n",
       "<text text-anchor=\"middle\" x=\"119.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">10 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 9&#45;&gt;10 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>9&#45;&gt;10</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M205.1223,-521.9735C189.1934,-509.1419 167.5964,-491.7443 150.0784,-477.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"151.8322,-474.551 141.849,-471.0034 147.4409,-480.0023 151.8322,-474.551\"/>\n",
       "<text text-anchor=\"middle\" x=\"192.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 17 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>17</title>\n",
       "<text text-anchor=\"middle\" x=\"227.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">17 (Najib)</text>\n",
       "</g>\n",
       "<!-- 9&#45;&gt;17 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>9&#45;&gt;17</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M227.5,-521.9735C227.5,-510.1918 227.5,-494.5607 227.5,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"231.0001,-481.0033 227.5,-471.0034 224.0001,-481.0034 231.0001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"239.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 22 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>22</title>\n",
       "<text text-anchor=\"middle\" x=\"227.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">22 (Umno)</text>\n",
       "</g>\n",
       "<!-- 17&#45;&gt;22 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>17&#45;&gt;22</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M227.5,-434.9735C227.5,-423.1918 227.5,-407.5607 227.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"231.0001,-394.0033 227.5,-384.0034 224.0001,-394.0034 231.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"239.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 11 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>11</title>\n",
       "<text text-anchor=\"middle\" x=\"820.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">11 (Wan)</text>\n",
       "</g>\n",
       "<!-- 11&#45;&gt;11 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>11&#45;&gt;11</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M860.6186,-982.7808C870.8323,-982.2694 878.5,-979.6758 878.5,-975 878.5,-972.0776 875.5048,-969.9686 870.7648,-968.673\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"871.014,-965.173 860.6186,-967.2192 870.0211,-972.1022 871.014,-965.173\"/>\n",
       "<text text-anchor=\"middle\" x=\"890.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 12 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>12</title>\n",
       "<text text-anchor=\"middle\" x=\"794.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">12 (Jan)</text>\n",
       "</g>\n",
       "<!-- 11&#45;&gt;12 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>11&#45;&gt;12</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M815.1128,-956.9735C811.5569,-945.0751 806.8278,-929.2508 802.7963,-915.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"806.0972,-914.5825 799.8803,-906.0034 799.3903,-916.5869 806.0972,-914.5825\"/>\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 18 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>18</title>\n",
       "<text text-anchor=\"middle\" x=\"894.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">18 (Razak)</text>\n",
       "</g>\n",
       "<!-- 11&#45;&gt;18 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>11&#45;&gt;18</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M835.8328,-956.9735C846.4494,-944.4919 860.7408,-927.6899 872.56,-913.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"875.3738,-915.8883 879.1868,-906.0034 870.0417,-911.3529 875.3738,-915.8883\"/>\n",
       "<text text-anchor=\"middle\" x=\"873.5\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 24 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>24</title>\n",
       "<text text-anchor=\"middle\" x=\"490.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">24 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 33&#45;&gt;24 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>33&#45;&gt;24</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M562.6447,-695.9735C549.3452,-683.2586 531.3562,-666.0603 516.6685,-652.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"518.9779,-649.384 509.3311,-645.0034 514.1406,-654.4437 518.9779,-649.384\"/>\n",
       "<text text-anchor=\"middle\" x=\"556.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 34 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>34</title>\n",
       "<text text-anchor=\"middle\" x=\"621.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">34 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 33&#45;&gt;34 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>33&#45;&gt;34</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M589.788,-695.9735C595.3122,-683.9585 602.677,-667.9401 608.9186,-654.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"612.2252,-655.5512 613.2226,-645.0034 605.8652,-652.627 612.2252,-655.5512\"/>\n",
       "<text text-anchor=\"middle\" x=\"615.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 15 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>15</title>\n",
       "<text text-anchor=\"middle\" x=\"752.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">15 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 16&#45;&gt;15 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>16&#45;&gt;15</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M802.4464,-695.9735C793.4924,-683.6085 781.4681,-667.0036 771.4613,-653.1847\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"774.2369,-651.05 765.5369,-645.0034 768.5673,-655.1556 774.2369,-651.05\"/>\n",
       "<text text-anchor=\"middle\" x=\"800.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 20 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>20</title>\n",
       "<text text-anchor=\"middle\" x=\"862.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">20 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 16&#45;&gt;20 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>16&#45;&gt;20</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M825.2384,-695.9735C831.7923,-683.8418 840.5512,-667.6287 847.9301,-653.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"851.1003,-655.4652 852.774,-645.0034 844.9416,-652.138 851.1003,-655.4652\"/>\n",
       "<text text-anchor=\"middle\" x=\"857.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 19 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>19</title>\n",
       "<text text-anchor=\"middle\" x=\"779.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">19 (dan)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;19 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>20&#45;&gt;19</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M845.3023,-608.9735C833.2833,-596.3752 817.0651,-579.3755 803.7375,-565.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"806.1108,-562.8228 796.6756,-558.0034 801.046,-567.6548 806.1108,-562.8228\"/>\n",
       "<text text-anchor=\"middle\" x=\"834.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 23 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>23</title>\n",
       "<text text-anchor=\"middle\" x=\"862.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">23 (,)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;23 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>20&#45;&gt;23</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M862.5,-608.9735C862.5,-597.1918 862.5,-581.5607 862.5,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"866.0001,-568.0033 862.5,-558.0034 859.0001,-568.0034 866.0001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"883\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 25 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>25</title>\n",
       "<text text-anchor=\"middle\" x=\"941.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">25 (Dr)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;25 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>20&#45;&gt;25</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M887.9844,-608.8488C894.6991,-603.4859 901.6854,-597.3421 907.5,-591 914.1542,-583.7421 920.44,-575.068 925.7545,-566.9496\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"928.7394,-568.7777 931.1114,-558.4518 922.8178,-565.0448 928.7394,-568.7777\"/>\n",
       "<text text-anchor=\"middle\" x=\"930.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 21 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>21</title>\n",
       "<text text-anchor=\"middle\" x=\"335.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">21 (Pemuda)</text>\n",
       "</g>\n",
       "<!-- 26 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>26</title>\n",
       "<text text-anchor=\"middle\" x=\"337.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">26 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 21&#45;&gt;26 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>21&#45;&gt;26</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M335.9144,-521.9735C336.1852,-510.1918 336.5446,-494.5607 336.8527,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"340.3553,-481.0812 337.0861,-471.0034 333.3571,-480.9202 340.3553,-481.0812\"/>\n",
       "<text text-anchor=\"middle\" x=\"349.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 27 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>27</title>\n",
       "<text text-anchor=\"middle\" x=\"337.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">27 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 26&#45;&gt;27 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>26&#45;&gt;27</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M337.5,-434.9735C337.5,-423.1918 337.5,-407.5607 337.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"341.0001,-394.0033 337.5,-384.0034 334.0001,-394.0034 341.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"349.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;21 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>24&#45;&gt;21</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M458.3839,-608.9735C434.5877,-595.617 401.9777,-577.3133 376.3304,-562.9177\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"378.0084,-559.8459 367.575,-558.0034 374.5821,-565.9501 378.0084,-559.8459\"/>\n",
       "<text text-anchor=\"middle\" x=\"434.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 30 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>30</title>\n",
       "<text text-anchor=\"middle\" x=\"490.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">30 (mempertikaikan)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;30 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>24&#45;&gt;30</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M490.5,-608.9735C490.5,-597.1918 490.5,-581.5607 490.5,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"494.0001,-568.0033 490.5,-558.0034 487.0001,-568.0034 494.0001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"501\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 29 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>29</title>\n",
       "<text text-anchor=\"middle\" x=\"460.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">29 (yang)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;29 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>30&#45;&gt;29</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M484.284,-521.9735C480.1811,-510.0751 474.7244,-494.2508 470.0726,-480.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"473.2768,-479.3161 466.7081,-471.0034 466.6592,-481.5981 473.2768,-479.3161\"/>\n",
       "<text text-anchor=\"middle\" x=\"497.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 31 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>31</title>\n",
       "<text text-anchor=\"middle\" x=\"576.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">31 (tindakan)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;31 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>30&#45;&gt;31</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M508.3193,-521.9735C520.7728,-509.3752 537.5771,-492.3755 551.3865,-478.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"554.1626,-480.5757 558.7036,-471.0034 549.1843,-475.6547 554.1626,-480.5757\"/>\n",
       "<text text-anchor=\"middle\" x=\"550.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 28 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>28</title>\n",
       "<text text-anchor=\"middle\" x=\"337.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">28 (Dusuki)</text>\n",
       "</g>\n",
       "<!-- 27&#45;&gt;28 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>27&#45;&gt;28</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M337.5,-347.9735C337.5,-336.1918 337.5,-320.5607 337.5,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"341.0001,-307.0033 337.5,-297.0034 334.0001,-307.0034 341.0001,-307.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"349.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 32 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>32</title>\n",
       "<text text-anchor=\"middle\" x=\"469.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">32 (kerajaan)</text>\n",
       "</g>\n",
       "<!-- 31&#45;&gt;32 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>31&#45;&gt;32</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M532.3438,-434.821C522.9329,-429.831 513.4469,-423.868 505.5,-417 497.5884,-410.1625 490.5267,-401.2346 484.7981,-392.7632\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"487.6649,-390.7493 479.3311,-384.2072 481.7662,-394.5184 487.6649,-390.7493\"/>\n",
       "<text text-anchor=\"middle\" x=\"542.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 36 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>36</title>\n",
       "<text text-anchor=\"middle\" x=\"590.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">36 (penuh)</text>\n",
       "</g>\n",
       "<!-- 31&#45;&gt;36 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>31&#45;&gt;36</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M579.4008,-434.9735C581.2967,-423.1918 583.8121,-407.5607 585.9688,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"589.4696,-394.4324 587.6029,-384.0034 582.5585,-393.3203 589.4696,-394.4324\"/>\n",
       "<text text-anchor=\"middle\" x=\"604.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 35 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>35</title>\n",
       "<text text-anchor=\"middle\" x=\"657.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">35 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 34&#45;&gt;35 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>34&#45;&gt;35</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M628.9592,-608.9735C633.931,-596.9585 640.5593,-580.9401 646.1767,-567.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"649.4608,-568.5818 650.0503,-558.0034 642.9927,-565.9053 649.4608,-568.5818\"/>\n",
       "<text text-anchor=\"middle\" x=\"679.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 40 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>40</title>\n",
       "<text text-anchor=\"middle\" x=\"968.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">40 (,)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;40 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>46&#45;&gt;40</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1115.3343,-697.5642C1098.14,-691.9713 1079.3293,-685.2756 1062.5,-678 1042.944,-669.5456 1022.0331,-658.4235 1004.9111,-648.7239\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1006.2368,-645.4493 995.8214,-643.5075 1002.7526,-651.5206 1006.2368,-645.4493\"/>\n",
       "<text text-anchor=\"middle\" x=\"1083\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 41 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>41</title>\n",
       "<text text-anchor=\"middle\" x=\"1057.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">41 (Najib)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;41 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>46&#45;&gt;41</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1147.8791,-695.9735C1130.9124,-683.0253 1107.8535,-665.4276 1089.274,-651.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1091.1635,-648.2878 1081.0906,-645.0034 1086.9168,-653.8525 1091.1635,-648.2878\"/>\n",
       "<text text-anchor=\"middle\" x=\"1140.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 45 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>45</title>\n",
       "<text text-anchor=\"middle\" x=\"1171.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">45 (sengaja)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;45 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>46&#45;&gt;45</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1171.5,-695.9735C1171.5,-684.1918 1171.5,-668.5607 1171.5,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1175.0001,-655.0033 1171.5,-645.0034 1168.0001,-655.0034 1175.0001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1200\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 61 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>61</title>\n",
       "<text text-anchor=\"middle\" x=\"1268.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">61 (.)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;61 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>46&#45;&gt;61</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1207.7759,-695.9949C1216.3794,-690.8331 1225.1757,-684.7504 1232.5,-678 1240.0709,-671.0223 1246.9804,-662.2002 1252.6767,-653.8634\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1255.6314,-655.7399 1258.1506,-645.4489 1249.7637,-651.9227 1255.6314,-655.7399\"/>\n",
       "<text text-anchor=\"middle\" x=\"1265\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 47 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>47</title>\n",
       "<text text-anchor=\"middle\" x=\"1375.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">47 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;47 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>46&#45;&gt;47</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1227.7223,-699.8233C1247.6142,-694.0344 1269.9058,-686.6492 1289.5,-678 1306.89,-670.3237 1325.1389,-659.8826 1340.3298,-650.4467\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1342.3078,-653.337 1348.8909,-645.0356 1338.5678,-647.4198 1342.3078,-653.337\"/>\n",
       "<text text-anchor=\"middle\" x=\"1329.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 62 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>62</title>\n",
       "<text text-anchor=\"middle\" x=\"1542.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">62 (Wan)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;62 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>64&#45;&gt;62</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1603.4239,-695.8061C1596.7253,-690.3493 1589.6151,-684.1679 1583.5,-678 1575.9414,-670.3761 1568.3763,-661.4313 1561.8395,-653.1796\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1564.5749,-650.9955 1555.6838,-645.2336 1559.0411,-655.2824 1564.5749,-650.9955\"/>\n",
       "<text text-anchor=\"middle\" x=\"1603.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 65 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>65</title>\n",
       "<text text-anchor=\"middle\" x=\"1627.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">65 (,)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;65 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>64&#45;&gt;65</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1627.5,-695.9735C1627.5,-684.1918 1627.5,-668.5607 1627.5,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1631.0001,-655.0033 1627.5,-645.0034 1624.0001,-655.0034 1631.0001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1648\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 83 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>83</title>\n",
       "<text text-anchor=\"middle\" x=\"1699.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">83 (.)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;83 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>64&#45;&gt;83</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1653.8871,-695.8513C1660.4537,-690.575 1667.1409,-684.4737 1672.5,-678 1678.3774,-670.9001 1683.5175,-662.3166 1687.6912,-654.2328\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1690.9105,-655.6155 1692.133,-645.0914 1684.6145,-652.5562 1690.9105,-655.6155\"/>\n",
       "<text text-anchor=\"middle\" x=\"1702\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 68 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>68</title>\n",
       "<text text-anchor=\"middle\" x=\"1813.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">68 (menjangka)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;68 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>64&#45;&gt;68</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1679.606,-697.0539C1694.9463,-691.5319 1711.6045,-685.0033 1726.5,-678 1743.5858,-669.9669 1761.7144,-659.6521 1776.9757,-650.4032\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1778.911,-653.3221 1785.6005,-645.1061 1775.2475,-647.3573 1778.911,-653.3221\"/>\n",
       "<text text-anchor=\"middle\" x=\"1772.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 43 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>43</title>\n",
       "<text text-anchor=\"middle\" x=\"1041.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">43 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 41&#45;&gt;43 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>41&#45;&gt;43</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1054.1848,-608.9735C1052.018,-597.1918 1049.1433,-581.5607 1046.6785,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1050.0621,-567.2053 1044.811,-558.0034 1043.1775,-568.4715 1050.0621,-567.2053\"/>\n",
       "<text text-anchor=\"middle\" x=\"1064.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 44 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>44</title>\n",
       "<text text-anchor=\"middle\" x=\"1151.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">44 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 41&#45;&gt;44 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>41&#45;&gt;44</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1076.9769,-608.9735C1090.7149,-596.2586 1109.2969,-579.0603 1124.4688,-565.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1127.0864,-567.3646 1132.0481,-558.0034 1122.3316,-562.2272 1127.0864,-567.3646\"/>\n",
       "<text text-anchor=\"middle\" x=\"1121.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 42 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>42</title>\n",
       "<text text-anchor=\"middle\" x=\"1024.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">42 (dan)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;42 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>43&#45;&gt;42</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1037.9776,-521.9735C1035.6754,-510.1918 1032.621,-494.5607 1030.0022,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1033.3708,-480.1465 1028.0179,-471.0034 1026.5007,-481.489 1033.3708,-480.1465\"/>\n",
       "<text text-anchor=\"middle\" x=\"1041.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 48 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>48</title>\n",
       "<text text-anchor=\"middle\" x=\"1296.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">48 (Perdana)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;48 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>47&#45;&gt;48</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1359.1311,-608.9735C1347.7972,-596.4919 1332.5403,-579.6899 1319.9225,-565.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1322.1615,-563.0538 1312.8479,-558.0034 1316.9793,-567.7595 1322.1615,-563.0538\"/>\n",
       "<text text-anchor=\"middle\" x=\"1352.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 57 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>57</title>\n",
       "<text text-anchor=\"middle\" x=\"1424.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">57 (lengkap)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;57 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>47&#45;&gt;57</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1385.6528,-608.9735C1392.4856,-596.8418 1401.6172,-580.6287 1409.3101,-566.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1412.5023,-568.4341 1414.3602,-558.0034 1406.4032,-564.9989 1412.5023,-568.4341\"/>\n",
       "<text text-anchor=\"middle\" x=\"1422.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 49 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>49</title>\n",
       "<text text-anchor=\"middle\" x=\"1134.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">49 (Menteri)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;49 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>48&#45;&gt;49</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1257.2676,-521.9203C1245.6529,-516.3664 1232.9762,-510.1046 1221.5,-504 1205.3713,-495.4206 1187.937,-485.3623 1172.9409,-476.4559\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1174.3315,-473.2095 1163.9523,-471.0829 1170.7399,-479.2179 1174.3315,-473.2095\"/>\n",
       "<text text-anchor=\"middle\" x=\"1233.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 50 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>50</title>\n",
       "<text text-anchor=\"middle\" x=\"1233.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">50 (,)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;50 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>48&#45;&gt;50</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1275.2721,-521.666C1269.7501,-516.3024 1264.0738,-510.204 1259.5,-504 1254.2172,-496.8342 1249.448,-488.4388 1245.4798,-480.5517\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1248.5275,-478.8119 1241.0561,-471.3 1242.2123,-481.8316 1248.5275,-478.8119\"/>\n",
       "<text text-anchor=\"middle\" x=\"1280\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 51 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>51</title>\n",
       "<text text-anchor=\"middle\" x=\"1316.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">51 (Tan)</text>\n",
       "</g>\n",
       "<!-- 48&#45;&gt;51 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>48&#45;&gt;51</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1300.644,-521.9735C1303.3793,-510.0751 1307.0171,-494.2508 1310.1182,-480.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1313.5318,-481.5333 1312.3613,-471.0034 1306.7098,-479.965 1313.5318,-481.5333\"/>\n",
       "<text text-anchor=\"middle\" x=\"1329\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 55 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>55</title>\n",
       "<text text-anchor=\"middle\" x=\"1413.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">55 (yang)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;55 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>57&#45;&gt;55</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1422.2208,-521.9735C1420.7311,-510.1918 1418.7548,-494.5607 1417.0602,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1420.5031,-480.4853 1415.7763,-471.0034 1413.5584,-481.3634 1420.5031,-480.4853\"/>\n",
       "<text text-anchor=\"middle\" x=\"1439.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 56 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>56</title>\n",
       "<text text-anchor=\"middle\" x=\"1516.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">56 (tidak)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;56 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>57&#45;&gt;56</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1444.4142,-521.7651C1450.5589,-516.099 1457.3308,-509.8143 1463.5,-504 1472.4123,-495.6003 1482.0855,-486.3364 1490.6986,-478.0346\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1493.1871,-480.4971 1497.9491,-471.0327 1488.3244,-475.4618 1493.1871,-480.4971\"/>\n",
       "<text text-anchor=\"middle\" x=\"1506\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 59 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>59</title>\n",
       "<text text-anchor=\"middle\" x=\"1651.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">59 (mengelirukan)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;59 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>57&#45;&gt;59</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1477.7315,-524.6166C1497.0424,-518.6626 1518.9177,-511.481 1538.5,-504 1560.8043,-495.4791 1584.9967,-484.7665 1605.2528,-475.3617\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1606.9848,-478.4154 1614.5574,-471.0056 1604.0168,-472.0758 1606.9848,-478.4154\"/>\n",
       "<text text-anchor=\"middle\" x=\"1595.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">ccomp</text>\n",
       "</g>\n",
       "<!-- 52 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>52</title>\n",
       "<text text-anchor=\"middle\" x=\"1316.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">52 (Sri)</text>\n",
       "</g>\n",
       "<!-- 51&#45;&gt;52 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>51&#45;&gt;52</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1316.5,-434.9735C1316.5,-423.1918 1316.5,-407.5607 1316.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1320.0001,-394.0033 1316.5,-384.0034 1313.0001,-394.0034 1320.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1328.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 53 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>53</title>\n",
       "<text text-anchor=\"middle\" x=\"1316.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">53 (Muhyiddin)</text>\n",
       "</g>\n",
       "<!-- 52&#45;&gt;53 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>52&#45;&gt;53</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1316.5,-347.9735C1316.5,-336.1918 1316.5,-320.5607 1316.5,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1320.0001,-307.0033 1316.5,-297.0034 1313.0001,-307.0034 1320.0001,-307.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1328.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 54 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>54</title>\n",
       "<text text-anchor=\"middle\" x=\"1316.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">54 (Yassin)</text>\n",
       "</g>\n",
       "<!-- 53&#45;&gt;54 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>53&#45;&gt;54</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1316.5,-260.9735C1316.5,-249.1918 1316.5,-233.5607 1316.5,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1320.0001,-220.0033 1316.5,-210.0034 1313.0001,-220.0034 1320.0001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1328.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 58 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>58</title>\n",
       "<text text-anchor=\"middle\" x=\"1596.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">58 (untuk)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;58 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>59&#45;&gt;58</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1640.104,-434.9735C1632.3607,-422.7252 1621.9874,-406.3165 1613.3013,-392.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1616.1835,-390.5857 1607.8814,-384.0034 1610.2667,-394.3262 1616.1835,-390.5857\"/>\n",
       "<text text-anchor=\"middle\" x=\"1642.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 60 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>60</title>\n",
       "<text text-anchor=\"middle\" x=\"1707.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">60 (rakyat)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;60 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>59&#45;&gt;60</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1663.1032,-434.9735C1670.9872,-422.7252 1681.5491,-406.3165 1690.3933,-392.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1693.4422,-394.3064 1695.9116,-384.0034 1687.5561,-390.5176 1693.4422,-394.3064\"/>\n",
       "<text text-anchor=\"middle\" x=\"1693.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 63 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>63</title>\n",
       "<text text-anchor=\"middle\" x=\"1542.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">63 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 62&#45;&gt;63 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>62&#45;&gt;63</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1542.5,-608.9735C1542.5,-597.1918 1542.5,-581.5607 1542.5,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1546.0001,-568.0033 1542.5,-558.0034 1539.0001,-568.0034 1546.0001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1554.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 66 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>66</title>\n",
       "<text text-anchor=\"middle\" x=\"1705.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">66 (beliau)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;66 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>68&#45;&gt;66</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1791.1223,-608.9735C1775.1934,-596.1419 1753.5964,-578.7443 1736.0784,-564.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1737.8322,-561.551 1727.849,-558.0034 1733.4409,-567.0023 1737.8322,-561.551\"/>\n",
       "<text text-anchor=\"middle\" x=\"1785.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 67 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>67</title>\n",
       "<text text-anchor=\"middle\" x=\"1816.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">67 (sudah)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;67 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>68&#45;&gt;67</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1814.1216,-608.9735C1814.5279,-597.1918 1815.0669,-581.5607 1815.529,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1819.0324,-568.1181 1815.8792,-558.0034 1812.0366,-567.8768 1819.0324,-568.1181\"/>\n",
       "<text text-anchor=\"middle\" x=\"1843\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 69 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>69</title>\n",
       "<text text-anchor=\"middle\" x=\"1919.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">69 (ada)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;69 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>68&#45;&gt;69</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1849.2808,-608.9356C1858.2362,-603.6991 1867.5489,-597.5959 1875.5,-591 1884.2302,-583.7577 1892.6617,-574.6817 1899.7673,-566.2013\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1902.5664,-568.3063 1906.1402,-558.3324 1897.1266,-563.9007 1902.5664,-568.3063\"/>\n",
       "<text text-anchor=\"middle\" x=\"1913\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 70 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>70</title>\n",
       "<text text-anchor=\"middle\" x=\"1805.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">70 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;70 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>69&#45;&gt;70</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1895.8791,-521.9735C1878.9124,-509.0253 1855.8535,-491.4276 1837.274,-477.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1839.1635,-474.2878 1829.0906,-471.0034 1834.9168,-479.8525 1839.1635,-474.2878\"/>\n",
       "<text text-anchor=\"middle\" x=\"1879.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 71 -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>71</title>\n",
       "<text text-anchor=\"middle\" x=\"1928.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">71 (balas)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;71 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>69&#45;&gt;71</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1921.3648,-521.9735C1922.5836,-510.1918 1924.2006,-494.5607 1925.5871,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1929.0899,-481.3105 1926.6376,-471.0034 1922.1271,-480.5901 1929.0899,-481.3105\"/>\n",
       "<text text-anchor=\"middle\" x=\"1961.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 73 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>73</title>\n",
       "<text text-anchor=\"middle\" x=\"2034.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">73 (Najib)</text>\n",
       "</g>\n",
       "<!-- 69&#45;&gt;73 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>69&#45;&gt;73</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1957.3276,-529.0585C1972.5402,-523.3402 1989.5148,-515.1576 2002.5,-504 2010.0688,-497.4965 2016.4428,-488.7509 2021.4604,-480.3399\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2024.5907,-481.9116 2026.3652,-471.4664 2018.4643,-478.5253 2024.5907,-481.9116\"/>\n",
       "<text text-anchor=\"middle\" x=\"2026.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 75 -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>75</title>\n",
       "<text text-anchor=\"middle\" x=\"1928.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">75 (tulisan)</text>\n",
       "</g>\n",
       "<!-- 71&#45;&gt;75 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>71&#45;&gt;75</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1928.5,-434.9735C1928.5,-423.1918 1928.5,-407.5607 1928.5,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1932.0001,-394.0033 1928.5,-384.0034 1925.0001,-394.0034 1932.0001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1939.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 74 -->\n",
       "<g id=\"node77\" class=\"node\">\n",
       "<title>74</title>\n",
       "<text text-anchor=\"middle\" x=\"1878.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">74 (mengenai)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;74 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>75&#45;&gt;74</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1918.14,-347.9735C1911.1677,-335.8418 1901.8498,-319.6287 1893.9999,-305.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1896.8642,-303.9295 1888.8468,-297.0034 1890.7951,-307.4175 1896.8642,-303.9295\"/>\n",
       "<text text-anchor=\"middle\" x=\"1921.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 77 -->\n",
       "<g id=\"node78\" class=\"node\">\n",
       "<title>77</title>\n",
       "<text text-anchor=\"middle\" x=\"2020.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">77 (berhubung)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;77 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>75&#45;&gt;77</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1947.5625,-347.9735C1961.0082,-335.2586 1979.1949,-318.0603 1994.0439,-304.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1996.601,-306.4173 2001.462,-297.0034 1991.7914,-301.3312 1996.601,-306.4173\"/>\n",
       "<text text-anchor=\"middle\" x=\"1990\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 72 -->\n",
       "<g id=\"node76\" class=\"node\">\n",
       "<title>72</title>\n",
       "<text text-anchor=\"middle\" x=\"977.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">72 (daripada)</text>\n",
       "</g>\n",
       "<!-- 72&#45;&gt;72 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>72&#45;&gt;72</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1034.0255,-982.4708C1044.5723,-981.5508 1052,-979.0605 1052,-975 1052,-972.3987 1048.9517,-970.4419 1043.9878,-969.1295\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1044.4541,-965.6596 1034.0255,-967.5292 1043.3438,-972.571 1044.4541,-965.6596\"/>\n",
       "<text text-anchor=\"middle\" x=\"1068\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 76 -->\n",
       "<g id=\"node79\" class=\"node\">\n",
       "<title>76</title>\n",
       "<text text-anchor=\"middle\" x=\"1910.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">76 (beliau)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;76 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>77&#45;&gt;76</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1997.7079,-260.9735C1981.3365,-248.0253 1959.0867,-230.4276 1941.1591,-216.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1943.2774,-213.4616 1933.2629,-210.0034 1938.935,-218.952 1943.2774,-213.4616\"/>\n",
       "<text text-anchor=\"middle\" x=\"1991.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 78 -->\n",
       "<g id=\"node80\" class=\"node\">\n",
       "<title>78</title>\n",
       "<text text-anchor=\"middle\" x=\"2020.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">78 (kesan)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;78 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>77&#45;&gt;78</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2020.5,-260.9735C2020.5,-249.1918 2020.5,-233.5607 2020.5,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2024.0001,-220.0033 2020.5,-210.0034 2017.0001,-220.0034 2024.0001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2031.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 79 -->\n",
       "<g id=\"node81\" class=\"node\">\n",
       "<title>79</title>\n",
       "<text text-anchor=\"middle\" x=\"2137.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">79 (positif)</text>\n",
       "</g>\n",
       "<!-- 77&#45;&gt;79 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>77&#45;&gt;79</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2044.7425,-260.9735C2062.2341,-247.9669 2086.0348,-230.269 2105.1473,-216.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2107.3524,-218.7791 2113.2886,-210.0034 2103.1755,-213.1618 2107.3524,-218.7791\"/>\n",
       "<text text-anchor=\"middle\" x=\"2105.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 80 -->\n",
       "<g id=\"node82\" class=\"node\">\n",
       "<title>80</title>\n",
       "<text text-anchor=\"middle\" x=\"2004.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">80 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 78&#45;&gt;80 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>78&#45;&gt;80</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2017.1848,-173.9735C2015.018,-162.1918 2012.1433,-146.5607 2009.6785,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2013.0621,-132.2053 2007.811,-123.0034 2006.1775,-133.4715 2013.0621,-132.2053\"/>\n",
       "<text text-anchor=\"middle\" x=\"2049.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 81 -->\n",
       "<g id=\"node83\" class=\"node\">\n",
       "<title>81</title>\n",
       "<text text-anchor=\"middle\" x=\"2140.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">81 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 79&#45;&gt;81 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>79&#45;&gt;81</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2138.1216,-173.9735C2138.5279,-162.1918 2139.0669,-146.5607 2139.529,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2143.0324,-133.1181 2139.8792,-123.0034 2136.0366,-132.8768 2143.0324,-133.1181\"/>\n",
       "<text text-anchor=\"middle\" x=\"2175.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 82 -->\n",
       "<g id=\"node84\" class=\"node\">\n",
       "<title>82</title>\n",
       "<text text-anchor=\"middle\" x=\"2004.5\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">82 (penuh)</text>\n",
       "</g>\n",
       "<!-- 80&#45;&gt;82 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>80&#45;&gt;82</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2004.5,-86.9735C2004.5,-75.1918 2004.5,-59.5607 2004.5,-46.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2008.0001,-46.0033 2004.5,-36.0034 2001.0001,-46.0034 2008.0001,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2024.5\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f091170fa58>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'KUALA LUMPUR: Ketua Penerangan BERSATU, Datuk Wan Saiful Wan Jan membidas kenyataan Datuk Seri Najib Razak dan Ketua Pemuda UMNO, Datuk Dr Asyraf Wajdi Dusuki yang mempertikaikan tindakan kerajaan melaksanakan sekatan pergerakan penuh. Beliau berkata, Najib dan Asyraf Wajdi sengaja memetik kenyataan Perdana Menteri, Tan Sri Muhyiddin Yassin yang tidak lengkap untuk mengelirukan rakyat. Wan Saiful berkata, beliau sudah menjangka ada kenyataan balas daripada Najib mengenai tulisan beliau berhubung kesan positif sekatan pergerakan penuh.'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, segment_sequence, mask_sequence, xlnet_sequence = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "            model.segment_ids: [segment_sequence],\n",
    "            model.input_masks: [mask_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(xlnet_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(xlnet_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'xlnet-base-dependency/model.ckpt'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'xlnet-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = dict(\n",
    "      is_training=False,\n",
    "      use_tpu=False,\n",
    "      use_bfloat16=False,\n",
    "      dropout=0.0,\n",
    "      dropatt=0.0,\n",
    "      init='normal',\n",
    "      init_range=0.1,\n",
    "      init_std=0.05,\n",
    "      clamp_len=-1)\n",
    "\n",
    "xlnet_parameters = xlnet.RunConfig(**kwargs)\n",
    "xlnet_config = xlnet.XLNetConfig(json_path='xlnet-base-29-03-2020/config.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:memory input None\n",
      "INFO:tensorflow:Use float type <dtype: 'float32'>\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(learning_rate, hidden_size_word)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from xlnet-base-dependency/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'xlnet-base-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(heads_pred, types_pred, heads, types, lengths,\n",
    "             symbolic_root=False, symbolic_end=False):\n",
    "    batch_size, _ = heads_pred.shape\n",
    "    ucorr = 0.\n",
    "    lcorr = 0.\n",
    "    total = 0.\n",
    "    ucomplete_match = 0.\n",
    "    lcomplete_match = 0.\n",
    "\n",
    "    corr_root = 0.\n",
    "    total_root = 0.\n",
    "    start = 1 if symbolic_root else 0\n",
    "    end = 1 if symbolic_end else 0\n",
    "    for i in range(batch_size):\n",
    "        ucm = 1.\n",
    "        lcm = 1.\n",
    "        for j in range(start, lengths[i] - end):\n",
    "\n",
    "            total += 1\n",
    "            if heads[i, j] == heads_pred[i, j]:\n",
    "                ucorr += 1\n",
    "                if types[i, j] == types_pred[i, j]:\n",
    "                    lcorr += 1\n",
    "                else:\n",
    "                    lcm = 0\n",
    "            else:\n",
    "                ucm = 0\n",
    "                lcm = 0\n",
    "\n",
    "            if heads[i, j] == 0:\n",
    "                total_root += 1\n",
    "                corr_root += 1 if heads_pred[i, j] == 0 else 0\n",
    "\n",
    "        ucomplete_match += ucm\n",
    "        lcomplete_match += lcm\n",
    "    \n",
    "    return ucorr / total, lcorr / total, corr_root / total_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1250/1250 [02:03<00:00, 10.15it/s]\n"
     ]
    }
   ],
   "source": [
    "arcs, types, roots = [], [], []\n",
    "real_Y, predict_Y = [], []\n",
    "\n",
    "for i in tqdm(range(0, len(test_X), batch_size)):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i: index]\n",
    "    batch_x = pad_sequences(batch_x,padding='post')\n",
    "    batch_y = test_Y[i: index]\n",
    "    batch_y = pad_sequences(batch_y,padding='post')\n",
    "    batch_depends = test_depends[i: index]\n",
    "    batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "    batch_segments = test_segments[i: index]\n",
    "    batch_segments = pad_sequences(batch_segments, padding='post', value = 4)\n",
    "    batch_masks = test_masks[i: index]\n",
    "    batch_masks = pad_sequences(batch_masks, padding='post', value = 1)\n",
    "    \n",
    "    tags_seq, heads = sess.run(\n",
    "        [model.logits, model.heads_seq],\n",
    "        feed_dict = {\n",
    "            model.words: batch_x,\n",
    "            model.segment_ids: batch_segments,\n",
    "            model.input_masks: batch_masks\n",
    "        },\n",
    "    )\n",
    "    \n",
    "    arc_accuracy, type_accuracy, root_accuracy = evaluate(heads - 1, tags_seq, batch_depends - 1, batch_y, \n",
    "            np.count_nonzero(batch_x, axis = 1))\n",
    "    arcs.append(arc_accuracy)\n",
    "    types.append(type_accuracy)\n",
    "    roots.append(root_accuracy)\n",
    "    predicted = pred2label(tags_seq)\n",
    "    real = pred2label(batch_y)\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_real_Y = []\n",
    "for r in real_Y:\n",
    "    temp_real_Y.extend(r)\n",
    "    \n",
    "temp_predict_Y = []\n",
    "for r in predict_Y:\n",
    "    temp_predict_Y.extend(r)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "          PAD    0.99976   1.00000   0.99988    339805\n",
      "            X    1.00000   0.99938   0.99969     62631\n",
      "          acl    0.84425   0.83292   0.83855      3202\n",
      "        advcl    0.64532   0.68824   0.66609      1684\n",
      "       advmod    0.95594   0.94239   0.94912      6700\n",
      "         amod    0.90791   0.90995   0.90893      4464\n",
      "        appos    0.84555   0.77299   0.80765      3088\n",
      "         case    0.98213   0.98372   0.98292     11117\n",
      "           cc    0.97966   0.97993   0.97979      3637\n",
      "        ccomp    0.48588   0.48315   0.48451       356\n",
      "     compound    0.91807   0.92646   0.92224     11381\n",
      "compound:plur    0.51163   0.66667   0.57895        33\n",
      "         conj    0.90245   0.89455   0.89849      5140\n",
      "          cop    0.97639   0.97639   0.97639       593\n",
      "        csubj    0.33333   0.16667   0.22222         6\n",
      "   csubj:pass    0.00000   0.00000   0.00000         1\n",
      "          dep    0.66500   0.73684   0.69908       361\n",
      "          det    0.94574   0.92229   0.93387      3912\n",
      "        fixed    0.82857   0.79452   0.81119       146\n",
      "         flat    0.95545   0.97113   0.96323     18638\n",
      "         iobj    0.00000   0.00000   0.00000         4\n",
      "         mark    0.91407   0.92163   0.91783      1812\n",
      "         nmod    0.87013   0.83202   0.85065      4429\n",
      "        nsubj    0.84932   0.87142   0.86023      6992\n",
      "   nsubj:pass    0.80982   0.80318   0.80648      1951\n",
      "       nummod    0.97935   0.94793   0.96338      4302\n",
      "          obj    0.90603   0.92001   0.91297      6351\n",
      "          obl    0.87106   0.87054   0.87080      5075\n",
      "    parataxis    0.46512   0.34739   0.39773       403\n",
      "        punct    0.99665   0.99622   0.99643     20881\n",
      "         root    0.90154   0.90740   0.90446     10000\n",
      "        xcomp    0.74568   0.75453   0.75008      1601\n",
      "\n",
      "     accuracy                        0.98035    540696\n",
      "    macro avg    0.78099   0.77564   0.77668    540696\n",
      " weighted avg    0.98030   0.98035   0.98029    540696\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(temp_real_Y, temp_predict_Y, digits = 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arc accuracy: 0.8481110435316473\n",
      "types accuracy: 0.8274148274750857\n",
      "root accuracy: 0.9210116457364005\n"
     ]
    }
   ],
   "source": [
    "print('arc accuracy:', np.mean(arcs))\n",
    "print('types accuracy:', np.mean(types))\n",
    "print('root accuracy:', np.mean(roots))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or '_seq' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'adam' not in n.name\n",
    "        and 'gradients/bert' not in n.name\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from xlnet-base-dependency/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-84-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 176 variables.\n",
      "INFO:tensorflow:Converted 176 variables to const ops.\n",
      "8206 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('xlnet-base-dependency', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-87-5eb1154ea798>:6: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "tf.set_random_seed(0)\n",
    "\n",
    "pb = 'xlnet-base-dependency/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "if 'bert' in pb:\n",
    "    inputs = ['Placeholder']\n",
    "    a = ['dense/BiasAdd']\n",
    "if 'xlnet' in pb:\n",
    "    inputs = ['Placeholder', 'Placeholder_1', 'Placeholder_2']\n",
    "    a = ['transpose_3']\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits', 'heads_seq'] + a, transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
